'''
Parse html files downloaded from LFC History to Liverbird database
'''

if __name__ == '__main__':
	pass

from BeautifulSoup import BeautifulSoup
import pickle


def unique(s):
    """Return a list of the elements in s, but without duplicates.

    For example, unique([1,2,3,1,2,3]) is some permutation of [1,2,3],
    unique("abcabc") some permutation of ["a", "b", "c"], and
    unique(([1, 2], [2, 3], [1, 2])) some permutation of
    [[2, 3], [1, 2]].

    For best speed, all sequence elements should be hashable.  Then
    unique() will usually work in linear time.

    If not possible, the sequence elements should enjoy a total
    ordering, and if list(s).sort() doesn't raise TypeError it's
    assumed that they do enjoy a total ordering.  Then unique() will
    usually work in O(N*log2(N)) time.

    If that's not possible either, the sequence elements must support
    equality-testing.  Then unique() will usually work in quadratic
    time.
    """

    n = len(s)
    if n == 0:
        return []

    # Try using a dict first, as that's the fastest and will usually
    # work.  If it doesn't work, it will usually fail quickly, so it
    # usually doesn't cost much to *try* it.  It requires that all the
    # sequence elements be hashable, and support equality comparison.
    u = {}
    try:
        for x in s:
            u[x] = 1
    except TypeError:
        del u  # move on to the next method
    else:
        return u.keys()

    # We can't hash all the elements.  Second fastest is to sort,
    # which brings the equal elements together; then duplicates are
    # easy to weed out in a single pass.
    # NOTE:  Python's list.sort() was designed to be efficient in the
    # presence of many duplicate elements.  This isn't true of all
    # sort functions in all languages or libraries, so this approach
    # is more effective in Python than it may be elsewhere.
    try:
        t = list(s)
        t.sort()
    except TypeError:
        del t  # move on to the next method
    else:
        assert n > 0
        last = t[0]
        lasti = i = 1
        while i < n:
            if t[i] != last:
                t[lasti] = last = t[i]
                lasti += 1
            i += 1
        return t[:lasti]

    # Brute force is all that's left.
    u = []
    for x in s:
        if x not in u:
            u.append(x)
    return u


class GameParser():
	def __init__(self):
		self.tags = ['Game Date ::', 'Competition ::', 'Stadium ::', 'Spectators ::']

		self.fulltags = ['LFC_id', 'Opposition', 'We scored', 'They scored', 'Game Date', 'Competition', 'Stadium', 'Spectators', 'Scorers','Players']
		

	def print_record(self):
#		size = min(len(self.keys), len(self	.record))
		print '\nStarting dump, size', len(self.record)		
		for i in range(0, len(self.fulltags)):
			print self.fulltags[i],self.record[i]

	def record_append(self, item):
#		print 'adding item', item
		self.record.append(item)
#		self.print_record()

	def get_player_id(self, item):
		sp = str(item)
		return int(sp[sp.find('id=')+3:sp.find('\" ')])		

	def parse_tags_and_score(self, item):
		if 'header_large' in str(item):
			string = item.renderContents().replace('&nbsp','').split(';')			
			if 'Liverpool' in string[0]:
				#self.record_append([string[4],string[1],string[3]])					
				self.record_append(string[4])					
				self.record_append(string[1])					
				self.record_append(string[3])					
				#print '\nOpponent is ', string[4], ' we scored ', string [1], ' they scored ', string [3]
			else:					
				self.record_append(string[0])					
				self.record_append(string[3])					
				self.record_append(string[1])					
				#print '\nOpponent is ', string[0], ' we scored ', string [3], ' they scored ', string [1]


		for tag in self.tags:
			if tag in str(item):	
				content = item.findNextSibling().renderContents()
				self.record_append(content)					
				#print tag, ' is ', content
	
	def parse_players(self, item):
		if 'Starting line-up' in str(item):
			self.player_list = []
			for i in item.findNext('table').findAll('tr'):
				p = i.findNext('a')					
				#print i.findNext().renderContents(),p.renderContents(), self.get_player_id(p)
				self.player_list.append([i.findNext().renderContents(),p.renderContents(), self.get_player_id(p),0,'e'])
				#print 'Player: n.', p.renderContents(), ', named', p.renderContents(), 'id', self.get_player_id(p)
				#p = player.findNext('a')					
				#self.player_list.append([player.renderContents(), p.renderContents(), self.get_player_id(p) ,0,'e'])
				#print 'Player: n.', player.renderContents(), ', named', p.renderContents(), 'id', self.get_player_id(p)
				#player = player.findNext('tr').findNext()
				
			if 'Subs' in str(item.findNext('font')):
				table = item.findNext('font').findNext('table')
				for i in table.findAll('tr'):
					p = i.findNext('a')					
					#print i.findNext().renderContents(),p.renderContents(), self.get_player_id(p)
					self.player_list.append([i.findNext().renderContents(),p.renderContents(), self.get_player_id(p),'e','e'])
	#				print table[i]#.findNext('a').renderContents()						
					#print 'Subs: n.', line.renderContents(), ', named', line.findNext('a').renderContents()
	

				
	def parse_scorers(self, item):
		#print 'before goals', item.findNext('font')		
		if 'Goals<br />' in str(item): 
			scorer_list = []
			for line in item.findNextSiblings('font'):
				p = line.findNext('a')

				
				if '\'' in line.renderContents().split(';')[1]:
					time = int(line.renderContents().replace('&nbsp','').split(';')[1].replace('\'',''))
				else:
					time = 'not known'

				
				scorer_list.append([p.renderContents(), self.get_player_id(p), time])
				#print p.renderContents(), self.get_player_id(p), time

			self.record_append(scorer_list)

	def parse_subs(self,item):
		if 'Substitutions' in str(item):
			for line in item.findNextSiblings('font'):
				if not 'Notes' in line:					
					p = line.findNext('a')
					p_in = self.get_player_id(p)
					p_out = self.get_player_id(p.findNext('a'))
					
					print line.renderContents()
					
					if '\'' in line.renderContents():
						time = int(line.renderContents().replace('&nbsp','').split(';')[-1].replace('\'',''))
					else:
						time = 'not known'
				
					for player in self.player_list:
						#print player							
						if player[2] == p_in:
							player[4] = time
						if player[2] == p_out:
							player[3] = time								
					#print 'sub was', p_in, p_out, time
				else:
					#print '3', self.player_list
					break



	def parse(self, file_index):
				
		try:
			page =  open("/home/aleks/Download/out/game_" + str(file_index)).read()

		except IOError:
			print 'File not found', "/home/aleks/Download/out/game_" + str(file_index)
			return

		self.record = []
		self.record_append(file_index)	
		
		soup = BeautifulSoup(page)
		for item in soup.html.body.findAll('font'):
			self.parse_tags_and_score(item)

			self.parse_players(item)

			self.parse_scorers(item)
			
			self.parse_subs(item)
			
		self.record_append(self.player_list)
								
		self.print_record()				


class GameDB:
	
	def __init__(self):
		self.all_players={}
		self.all_liverbird_players={}

		self.player_pickle_file = 'ppf.bin'
		self.liverbird_player_pickle_file = 'lppf.bin'
		
		self.tourn_dict = { 'European Cup Winners':	4653,\
							'1st Division':			4654,\
							'2nd Division':			4655,\
							'Centenary Trophy':		5195,\
							'Champions League':		310,\
							'Charity Shield': 		4672,\
							'Community Shield':		4673,\
							'Europa League':		3243,\
							'European Cup':			4675,\
							'European Fairs Cup':	4676,\
							'European Super Cup':	4677,\
							'FA Cup'			:	154,\
							'Lancashire League':	4678,\
							'League Cup':			319,\
							'Premier League':		69,\
							'Screen Sport':			4679,\
							'Sheriff of London':	4674,\
							'Test Match':			4680,\
							'UEFA Cup':				4682,\
							'World Club':			4681
						 }



	def load_dump(self, file):
		print 'loading base'
		self.dump = open(file, 'r').readlines()

	def find_all_players(self):
		for line in self.dump:
			pl_list = line.split('Players\',')[1]
			for player in pl_list.split('], ['):
				if '[]' in player:
					continue
				name = player.split(', ')[1]
				id = player.split(', ')[2]
				if self.all_players.has_key(id):
					if self.all_players[id] == name:
						pass
					else:
						print 'Name confict %s or %s under id %d' % (self.all_players[id], name, id) 
				else:
					self.all_players[id] = name
			
	def print_all_players_old(self):
		keylist = self.all_players.keys()
		keylist.sort()
		print '\nprinting list'
		f = open('pl_list.txt','w')
		for key in keylist:
 			#print "%s: %s" % (key, self.all_players[key])
 			f.write(' '.join([key, self.all_players[key],'\n']))

	def print_all_players(self):
		print self.all_players

	def store_all_players(self):
		pickle.dump(self.all_players, open(self.player_pickle_file,'w'))

	def restore_all_players(self):
		self.all_players = pickle.load(open(self.player_pickle_file,'r'))

	def store_all_liverbird_players(self):
		pickle.dump(self.all_liverbird_players, open(self.liverbird_player_pickle_file,'w'))

	def restore_all_liverbird_players(self):
		self.all_liverbird_players = pickle.load(open(self.liverbird_player_pickle_file,'r'))


	def read_liverbird_players(self, file):
		list = open(file, 'r').read()
		player_list = list.split('),(')
		for player in player_list:
			items = player.split(',')
			id = items[1]
			rus_name = (items[-2][1:-1]+' '+items[2][1:-1])
			eng_name = (items[ 3][1:-1]+' '+items[4][1:-1])
			
			print 'found id', id
			self.all_liverbird_players[id] = [rus_name, eng_name]

	def compare_player_lists(self):
		self.restore_all_players()
		self.restore_all_liverbird_players()
		missing = 0
		
		f = open('comp_players.txt', 'w')
		
		''' LFC player_id, LFC name, LB name, LB ID'''
		
		for lfc_player in self.all_players:
			record = []
			found = False
			record.append(lfc_player)
			
			l_name = self.all_players[lfc_player].decode('string_escape')[1:-1]
			record.append(l_name)

			for our_player in self.all_liverbird_players:
				lb_name = self.all_liverbird_players[our_player][1].decode('string_escape')
		
				#print "cmp <%s>/<%s>" % (l_name, lb_name)
				if l_name in lb_name:
					found = True
					record.append(l_name)
					record.append(our_player)
					break;
			else:		
				#print 'not found <%s>' % (l_name)
				missing += 1
				record.append('-->')
				family_name = l_name.split(' ')[-1]
				for our_player in self.all_liverbird_players:
					lb_name = self.all_liverbird_players[our_player][1].decode('string_escape')
					
					lb_fname = lb_name.split(' ')[-1]
					#print 'cmp', lfc_fname, family_name
					if lb_fname == family_name:
						#print "option <%s>" % (lb_name)
						record.append(lb_name)
						record.append(our_player)
				
				#if missing > 2:
				#	return
			f.write(', '.join(record))
			f.write('\n')
				
		print "missing ", missing 
		f.close()


	
	def compare_player_lists_back(self):
		self.restore_all_players()
		self.restore_all_liverbird_players()
		missing = 0
		
		
		for our_player in self.all_liverbird_players:
			lb_name = self.all_liverbird_players[our_player][1].decode('string_escape')
		
			for lfc_player in self.all_players:
				l_name = self.all_players[lfc_player].decode('string_escape')[1:-1]

				if l_name in lb_name:
					break;
			else:		
				print 'not found <%s>' % (lb_name)
				missing +=1
				
				
		print "missing ", missing 
	
	
	def find_all_tournaments(self):
		lc = 0
		for line in self.dump:
			lc+=1
			c = line.split('Competition\',')[1].split(']')[0].replace('\'','')[1:]
			for key in self.tourn_keys:
				if key in c:
					break
			else:
				print 'Not found', c, 'line', lc
				return

	def generate_game_list(self):
		lc = 0
		output = open('results_dump.txt','w')
		for line in self.dump:
			lc+=1
			
			if lc>=5099:
				return
			
			record = []
			line = line.decode('string_escape')
			
			lfc_id = line.split('LFC_id\',')[1].split(']')[0][1:]
			''' Processing Opposition'''
			opp = line.split('Opposition\',')[1].split(']')[0][1:-1]
			for key in self.opp_dict:
				if key in opp:
					record.append(self.opp_dict[key])
					break
			else:
				print 'nf', opp
				return
			
			''' Processing Stadium'''
			st = line.split('Stadium\',')[1].split(']')[0][2:-1]
			for key in self.stad_dict:
				if key in st:
					record.append(self.stad_dict[key])
					break
			else:
				print 'nf', st
				return

			


			''' Processing Competition'''
			comp = line.split('Competition\',')[1].split(']')[0][1:-1]
			for key in self.tourn_dict:
				if key in comp:
					record.append(self.tourn_dict[key])
					break
			else:
				print 'Not found', comp, 'line', lc
				return

			date =  line.split('Game Date\',')[1].split(']')[0][2:-1]
			d = date.split('.')
			dd = '%s-%s-%s:T00:00:00' % (d[2],d[1],d[0])
			att =  line.split('Spectators\',')[1].split(']')[0][2:-1].replace(',','')
			if 'Not yet' in att:
				att = ''
			we =  line.split('We scored\',')[1].split(']')[0][2:-1]
			them =  line.split('They scored\',')[1].split(']')[0][2:-1]
			
			''' Home\Away switch '''
			if record[1] == '112':
				h_a = 'H'
			else:
				if record[1] == '4935':
					h_a = 'N'
				else:
					h_a = 'A'
				
					
			''' Opposition, home_or_away, stadium, competition, date, attendance, we scored, they scored '''
			output.write('array(\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\"),\n' \
						% (lfc_id, record[0], h_a ,record[1], record[2], dd, att, we, them))
			#if lc > 10:
			#	return
			
 
	def find_all_opponents(self):
		opp_list=[]
		for line in self.dump:
			c = line.split('Opposition\',')[1].split(']')[0].replace('\'','')[1:]
			if not c in opp_list:
				opp_list.append(c)
		
		#print len(opp_list)#sorted(opp_list)
		opp_list_l_short=[]
		opp_list_l_long=[]

		list = open('1', 'r').read()
		lvbd_list = list.split('),(')
		for opp in lvbd_list:
			items = opp.split(',')
			id = items[1]
			eng_long = items[4][1:-1]
			eng_short = items[5][1:-1]
			opp_list_l_long.append(eng_long)
			opp_list_l_short.append(eng_short)
			#print id, eng_name


		print 'total', len(opp_list)
		found1=0
		found2=0
		for team in opp_list:
			#print team
			for item in opp_list_l_long:
				if team in item:
					found1+=1
					print '1 Team found %s // in %s' % (team, item)
					break
			else:
				for item in opp_list_l_short:
					if item in team: 
						found2+=1
						print '2 Team found %s // in %s' % (team, item)
						break
				else:
					print team
			#x = get_close_matches(team, opp_list_l_short, 1, 0.8)
			#if x:
			#	print 'not found', team #, 'maybe its', x 
			
		print found1, found2

	def find_all_stadiums(self):
		stad_list=[]
		for line in self.dump:
			st = line.split('Stadium\',')[1].split(']')[0][2:-1]
			stad_list.append(st)
		
		lfc_list = sorted(unique(stad_list))

		l_dump = open('2','r').readlines()
		l_stads = []
		for line in l_dump:
			l_stads.append(line.split('/')[1][1:-1])

		for st in lfc_list:
			for l_st in l_stads:
				if st in l_st:
					#print 'found', st
					break
			else:
				print st 

	def parse_liverbird_sql(self, file):
		stream = open(file, 'r').read()
		out ={}
		for item in stream.split('),('):
			set = item.split(',')
			out[set[3][1:-1]] = set[1]

		return out

	def parse_liverbird_sql2(self, file):
		stream = open(file, 'r').read()
		out ={}
		for item in stream.split('),('):
			set = item.split(',')
			out[set[4][1:-1]] = set[1]

		return out

	def parse_liverbird_sql3(self, file):
		stream = open(file, 'r').read()
		out ={}
		for item in stream.split('),('):
			set = item.split(',')
			out[set[5][1:-1]] = set[1]

		return out

	def create_stadium_list_lb(self, file):
		self.stad_keys = self.parse_liverbird_sql(file)
		#print self.stad_keys

	def create_stadium_list_his(self):
		self.stad_list=[]
		for line in self.dump:
			st = line.split('Stadium\',')[1].split(']')[0][2:-1]
			if not st in self.stad_list:
				self.stad_list.append(st.decode('string_escape'))

	def create_opp_list_lb(self, file):
		self.opp_keys = self.parse_liverbird_sql2(file)
		self.opp_keys_sh = self.parse_liverbird_sql3(file)
		#print self.opp_keys_sh 

	def create_opp_list_his(self):
		self.opp_list=[]
		for line in self.dump:
			c = line.split('Opposition\',')[1].split(']')[0].replace('\'','')[1:]
			if not c in self.opp_list:
				self.opp_list.append(c.decode('string_escape'))
	
	def create_comp_list_lb(self, file):
		self.comp_keys = self.parse_liverbird_sql(file)
		#print self.comp_keys

	def write_merge_stad(self, file):
		out = open(file, 'w')
		for st in self.stad_list:

			id = 'NA'
			for key in self.stad_keys.keys():
				#print key
				if key in st:
					#print 'f', key
					id = self.stad_keys[key]
					break
			else:
				for key in self.stad_keys.keys():
					#print key
					if st in key:
						print 'f', key
						id = self.stad_keys[key]
						break	
			
			out.write ('%s,%s\n' % (st, id))
	
	def read_merge_stad(self, file):
		self.stad_dict = {}
		xd = {}
		for line in open(file, 'r').readlines():
			items = line.decode('string_escape').split(',')
			self.stad_dict[items[0]] = items[1][:-1]
			
			a = items[1][:-1]
			b = items[0]
			
			if xd.has_key(a):
				print "double", a, xd[a],'-' , b
			
			xd[a] = b
		#print self.stad_dict  

	def write_merge_opp(self, file):
		out = open(file, 'w')
		for opp in self.opp_list:

			id = 'NA'
			for key in self.opp_keys_sh.keys():
				#print key
				if key in opp:
					print 'f %s in %s' % (key, opp)
					id = self.opp_keys_sh[key]
					break
			else:
				for key in self.opp_keys.keys():
					#print key
					if opp in key:
						print 'f %s in %s' % (opp, key)
						id = self.opp_keys[key]
						break	
			
			out.write ('%s,%s\n' % (opp, id))
	
	def read_merge_opp(self, file):
		self.opp_dict = {}
		for line in open(file, 'r').readlines():
			items = line.decode('string_escape').split(',')	
			self.opp_dict[items[0]] = items[1][:-1]
			
		#print self.opp_dict

	def read_edited_liverbird_players(self, file):
		self.np_dict = {}
		for l in open(file, 'r').readlines():
			xid =  l.split(',')[0]
			lb_id = l.split(',')[3][:-1]
			lb_name = l.split(',')[2]
			
			self.np_dict[xid] = lb_id

	def gen_pl_list(self):

		p.load_dump('full_dump.txt')
		p.read_edited_liverbird_players('comp_players_edited.txt')
		output = open('pl_list_x1.txt','w')
		
		lc = 0
		for line in self.dump:
			lc+=1
			
#			if lc>=2:
#				return

			if lc>=5099:
				return
			
			record = []
			line = line.decode('string_escape')

			lfc_id = line.split('LFC_id\',')[1].split(']')[0][1:]

			
			''' Processing Players'''
			pl_list = line.split('Players\',')[1].split(']]')[0]#[1:]
			pl_list_item = pl_list[1:].split('],')
			pos = 0
			record = []
			for item in pl_list_item:
				pos += 1
				
				x = item.split(',')
				shirt = x[0][3:-1]
				p_id = x[2][1:]
				pl_id = self.np_dict[p_id][1:]

				if x[3][2:-1] == 'e':
					came_on = 100500
				else:
					came_on = x[3][1:]

				
				if x[4][2:-1] == 'e':
					came_off = 100500
				else:
					came_off = x[4][1:]

#				print lfc_id, pos, shirt, pl_id, came_on, came_off
				
#				print item
			
			
			
				''' match id, player teamsheet number, shirt number, player id, came on, came off, captain, yellow card, second yellow, red card '''
				output.write('array(\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\"),\n' \
							% (lfc_id, pos, shirt, pl_id, came_on, came_off, 'NULL', 'NULL', 'NULL', 'NULL' ))
			#if lc > 10:
			#	return

		

p = GameDB()

p.gen_pl_list()


#p.compare_player_lists_back()

#p.load_dump('full_dump.txt')
#p.create_comp_list_lb('compet.sql')

#p.create_stadium_list_lb('stadium.sql')
#p.create_opp_list_lb('opposition.sql')

#p.create_stadium_list_his()
#p.create_opp_list_his()


#p.write_merge_stad('stad_merge.txt')
#p.read_merge_stad('stad_merge.txt')

#p.write_merge_opp('opp_merge.txt')
#p.read_merge_opp('opp_merge.txt')


#p.generate_game_list()

print 'done'